<?php
/**
 * 采集功能
 */
namespace app\admin\controller;

use think\Controller;
use think\Request;
use QL\QueryList;
class Collection extends Controller
{
    /**
     * 显示资源列表
     *
     * @return \think\Response
     */
    public function list()
    {
        $data = QueryList::get('https://www.baidu.com/s?wd=QueryList')
        // 设置采集规则
        ->rules([ 
            'title'=>array('h3','text'),
            'link'=>array('h3>a','href')
        ])
        ->queryData();
  
    dump($data);
    }
    public function index(){
        $res=db('collection')->alias('c')->field('c.*,m.model_name')->join('models m','m.id=c.model_id')->order('c.id desc')->paginate(15);
        $this->assign('res',$res);
        return view();
    }

    /**
     * 显示创建资源表单页.
     *
     * @return \think\Response
     */
    public function create()
    {
        $model=db('models')->select();
        $this->assign('model',$model);
       return view();
    }

    /**
     * 保存新建的资源
     *
     * @param  \think\Request  $request
     * @return \think\Response
     */
    public function save()
    {
        if (request()->isPost()) {
            $_data=input('post.');
            $rules=array(//列表页的采集规则
                "list_url"   =>$_data['list_url'] ,
                "start_page" => $_data['start_page'] ,
                "end_page"   => $_data['end_page'] ,
                "step"       =>$_data['step'] , 
                "range"      =>$_data['range'] ,
                'list_rules' =>array(
                    'url'    =>$_data['url'],
                    'litpic' =>$_data['litpic'],
                    ),
            );
            $data['note_name']      =$_data['note_name'];
            $data['model_id']       =$_data['model_id'];
            $data['output_encoding']=$_data['output_encoding'];
            $data['input_encoding'] =$_data['input_encoding'];
            $data['list_rules']     =json_encode($rules);
            $data['addtime']        =time();
            $id=db('collection')->insertGetId($data);//返回主键id
            if ($id) {
                $res=db('collection')->field('model_id')->find($id);
                $this->redirect('collection/rules',['model_id'=>$res['model_id'],'id'=>$id]);
            }else{
                $this->error('添加节点失败');
            }
        }
    }

    /**
     * 内容规则
     *
     * @param  int  $id
     * @return \think\Response
     */
    public function rules()
    {
        $id=input('id');
        $modelid=input('model_id');
        if (request()->isPost()) {
          $data=input('post.');
          $rules=array();
          if($data){
              foreach ($data as $k => $v) {
                      $rules[$k][0]=$v[0];
                      $rules[$k][1]=$v[1];
                      $rules[$k][2]=$v[2];
                      array_values($rules[$k]);
                  
              }
          }
          $rules=json_encode($rules);
          $res=db('collection')->where('id',$id)->update(['item_rules'=>$rules]);
          if ($res) {
              $this->success('节点添加成功','index');
          }
          $this->error('节点添加失败');
        }
        
        $fieldes=db('fields')->field('cname,ename')->where('model_id',$modelid)->select();
        $this->assign('fieldes',$fieldes);
        return view();
    }

    /**
     * 显示编辑资源表单页.
     *
     * @param  int  $id
     * @return \think\Response
     */
    public function edit($id)
    {   
        if (request()->isPost()) {
            $_data=input('post.');
            $rules=array(//列表页的采集规则
                "list_url"   =>$_data['list_url'] ,
                "start_page" => $_data['start_page'] ,
                "end_page"   => $_data['end_page'] ,
                "step"       =>$_data['step'] , 
                "range"      =>$_data['range'] ,
                'list_rules' =>array(
                    'url'    =>$_data['url'],
                    'litpic' =>$_data['litpic'],
                    ),
            );
            $contnets=array(//内容采集规则
                'keywords'    =>$_data['keywords'],
                'description' =>$_data['description'],
                'writer'      =>$_data['writer'],
                'source'      =>$_data['source'],
                'content'     =>$_data['content'],
                'img'         =>$_data['img'],
                'title'         =>$_data['title'],
            );
            $data['note_name']      =$_data['note_name'];
            $data['model_id']       =$_data['model_id'];
            $data['output_encoding']=$_data['output_encoding'];
            $data['input_encoding'] =$_data['input_encoding'];
            $data['list_rules']     =json_encode($rules);
            $data['item_rules']     =json_encode( $contnets);
            $edit=db('collection')->where('id',$_data['id'])->update($data);
            if ($edit) {
              $this->success('采集节点编辑成功','index');
            }
            $this->error('采集节点编辑失败');
        }
        $note=db('collection')->alias('c')->field('c.*,m.model_name')->join('models m','m.id=c.model_id')->where('c.id',$id)->find();
        $modelid=$note['model_id'];    
        $fieldes=db('fields')->field('cname,ename')->where('model_id',$modelid)->select();    
        $list=json_decode($note['list_rules'],true);
        $item=json_decode($note['item_rules'],true);
        $this->assign([
            'note'   =>$note,
            'fieldes'=>$fieldes,
            'modelid'=>$modelid,
            'list'   =>$list,
            'item'   =>$item,
        ]);
        return view();
    }

    /**
     * 
     *
     * @param  \think\Request  $request
     * @param  int  $id
     * @return \think\Response
     */
    public function show($id)
    {
       
        $cateRes=model('Cate')->cateSort();
        $note=db('collection')->field('model_id,note_name')->find($id);
        $modelname=$note['note_name'];
        $modelid=$note['model_id'];
        $this->assign([
            'id'        =>$id,
            'cateRes'   =>$cateRes,
            'modelname' =>$modelname,
            'modelid'   =>$modelid,
        ]);
        return view();
    }

    /**
     * 删除指定资源
     *
     * @param  int  $id
     * @return \think\Response
     */
    public function delete($id)
    {
      if (request()->isAjax()) {
          $res=db('collection')->delete($id);
          if ($res) {
              return json(['code'=>1,'msg'=>'删除成功']);
          }
          return json(['code'=>2,'msg'=>'删除失败']);
      }
    }
    /**
     * 开始采集
     *
     * @return void
     */
    public function doCollect(){
        $id=input('id');
        $res=db('collection')->where('id',$id)->find();
        $list_rules=json_decode($res['list_rules'],true);//列表采集规则
        //输出编码
        $outputEncoding=$res['output_encoding'];
        //输入编码
        $inputEncoding=$res['input_encoding'];
        //内容页采集规则
        $itemRules=json_decode($res['item_rules'],true);
        //采集开始页码
        $startPage=$list_rules['start_page'];
        //采集结束页码
        $endPage=$list_rules['end_page'];
        //页码步长
        $step=$list_rules['step']; 
        //采集范围
        $range=$list_rules['range']; 
        $rules=array(
            'url'   =>array($list_rules['list_rules']['url'],'href'),
            'litpic'=>array($list_rules['list_rules']['litpic'],'src'),
        );
        $url=$list_rules['list_url'];
        $_listUrl=[];//转换为实际页码的列表网址
        //处理采集列表网址
        for ($i=$startPage; $i <= $endPage; $i=$i+$step) { 
            $_listUrl[]=str_replace('(*)', $i, $url);
        }
         //循环采集数据
         $_data=[];
         foreach ($_listUrl as $k => $url) {
             $_data[] =QueryList::get($url)//列表地址
             ->rules($rules)//规则
             ->range($range)//切片选择器，范围
            // ->encoding($outputEncoding,$inputEncoding)//编码
             ->query()
             ->queryData();
         }
        // 列表采集数据重构
        $listData=[];
        foreach ($_data as $k => $v) {  
            foreach ($v as $k1 => $v1) {
                $listData[]=$v1;
                }  
        }
        // halt( $listData);
        //内容采集规则
        // $items=array(
        //     'title'=>array('div.good_right>h1','text'),
        // );
         //内容数据采集结果集
         $_dataItem=[];
         foreach ($listData as $k => $v) {
             $_dataItem[] = QueryList::get($v['url'])//列表地址
             ->rules($itemRules)//规则
             ->encoding($outputEncoding,$inputEncoding)
             ->query()
             ->queryData();
             $_dataItem[$k][0]['url']=$v['url'];//手动添加url到数组，目的是写入临时表中
             $_dataItem[$k][0]['litpic']=$v['litpic'];//列表采集的缩略图
         }
          //数组重构
        $dataItem=[];
        foreach ($_dataItem as $k => $v) {
            foreach ($v as $k1 => $v1) {
                $dataItem[]=$v1;  
            }
        }
        // halt($dataItem);
        foreach ($dataItem as $k => $v) {
            $data['nid']=$id;
            if (isset($v['title'])) {
                $data['title']=$v['title'];
            }
            // //防止重复采集
            // $reData=db('html')->where(array('title'=>$data['title']))->find();
            // if($reData){
            //     continue;
            // }// 
            if (isset($v['url'])) {
                $data['url']=$v['url'];
            }
            $data['addtime']=time();
            $data['result']=json_encode($v);
            db('html')->insert($data);
        }
        //节点最后采集时间
        db('collection')->where(array('id'=>$id))->update(['lasttime'=>time()]);
        echo 1;//采集完成
       
    }
    /**
     * 数据导出
     *
     * @return void
     */
    public function exportdata(){
        $noteId=input('id');//当前节点id
        $cateId=input('cate_id');//要导出数据所属栏目
        $cate=db('cate')->field('model_id')->find($cateId);
        $modelId=$cate['model_id'];//动态获取模型id
        $model=db('models')->field('table_name')->find($modelId);
        $tableName=$model['table_name'];
        $data=db('html')->field('id,export,result')->where(array('export'=>0,'nid'=>$noteId))->select();//要导出的数据
        // dump($data); die;
        $arr=array('title','keywords','description','writer','source','content','litpic','url');
        $_archives=[];//主表元素数组
        $_addTable=[];//附加表元素数组
        $i=0;
        foreach ($data as $k => $v) {
            $_data=json_decode($v['result'],true);
            foreach ($_data as $k1 => $v1) {
                if(in_array($k1, $arr)){
                    $_archives[$k1]=$v1;
                    if($k1=='url'){
                        unset($_archives[$k1]);
                    }
                }else{
                    
                     $_addTable[$k1]=$v1;
                }
            }
            $_addTable['number']=$_addTable['bianhao'];
            $_addTable['format']=$_addTable['geshi'];
            $_addTable['volume']=$_addTable['tiji'];
            unset($_addTable['bianhao']);
            unset($_addTable['geshi']);
            unset($_addTable['tiji']);
           $_archives['cid']=$cateId;
           $_archives['model_id']=$modelId;
           $addId=db('archives')->insertGetId($_archives);
           $_addTable['aid']=$addId;
           db($tableName)->insert($_addTable);
           db('html')->where(array('id'=>$v['id']))->update(['export'=>1]);
           //批量导出
           $i++;
           if(($i%6)==0){
            sleep(3);
           }
        }
        echo 1;//数据导出完成
    }
}
